#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :cidianwang_gushiwen.py
# @Time      :2023/11/10 
# @Author    :CL
# @email     :1037654919@qq.com
# 爬取 https://www.cidianwang.com/gushiwen/
from bs4 import BeautifulSoup
import requests
from retrying import retry
from utils import mongo_manager, get_kuai_proxy

cidainwang_gushiwen = mongo_manager("cidainwang_gushiwen", db='public_data')
cidainwang_gushiwen_tag = mongo_manager("cidainwang_gushiwen_tag", db='public_data')

headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Pragma": "no-cache",
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
    "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Linux\""
}
cookies = {
    "Hm_lvt_e0eec4afa8ab3236c032068920f81d60": "1699509492",
    "Hm_lpvt_e0eec4afa8ab3236c032068920f81d60": "1699509492"
}

#获取tag
def get_gushiwen_tag(url = 'https://www.cidianwang.com/gushiwen/'):
    lists=[]
    response = requests.get(url,  headers=headers, cookies=cookies) #proxies=get_kuai_proxy(),
    response.encoding = "utf-8"
    print(url, response)
    soups = BeautifulSoup(response.text, 'lxml')
    k_ml = soups.find('div', class_='left').find_all('div',class_='k_ml')
    for ml in k_ml:
        types = ml.find('div',class_='fl1').get_text().replace('：', '')
        datas = ml.find_all('li')
        for data in datas:
            name = data.find('a').get_text()
            href = 'https://www.cidianwang.com' + data.find('a').get('href')
            lists.append({'type': types,'name': name, 'href': href})
    return lists
def get_gushi(url='https://www.cidianwang.com/gushiwen/xianqin_1.htm'):
    lists = []
    try:
        response = requests.get(url, headers=headers, cookies=cookies) #,proxies=get_kuai_proxy()
        response.encoding = "utf-8"
        print(url, response)
        soups = BeautifulSoup(response.text, 'lxml')

        datas = soups.find('div', class_='left').find_all('div',recursive=False)[-1].find_all('li')
        # print(datas)
        for data in datas:
            name = data.find('a').get_text()
            href = 'https://www.cidianwang.com' + data.find('a').get('href')
            sczz = data.find('div', class_='sczz').get_text()  # 作者：柳宗元，朝代：唐代 标签：忧国忧民
            intru = data.find('div',class_='finfo').get_text().replace(sczz, '').strip()

            lists.append({'name': name, 'href': href,'info': sczz, 'intru': intru})
    except BaseException as e:
        print(e)
    return lists


def down_gushi_list():  # 下载gushiwen
    seeds = cidainwang_gushiwen_tag.findAll({'type': '朝代',"status":None})
    for tag in seeds:
        if tag['type'] != '朝代':
            continue

        print(f'begin {tag["href"]}')
        for i in range(0, 1000):
            if i == 0:
                url = tag["href"] # https://www.cidianwang.com/gushiwen/xianqin.htm https://www.cidianwang.com/gushiwen/xianqin_1.htm
            else:
                url = tag["href"].replace('.htm', f'_{i}.htm')
            lists = get_gushi(url)
            print(len(lists))
            for ll in lists:
                ll['_id'] = ll['href']
                # ll['type'] = tag
                try:
                    cidainwang_gushiwen.insertOne(ll)
                except Exception as e:
                    print(e)
            if len(lists) < 20:
                break
        cidainwang_gushiwen_tag.updateOne({"_id": tag["_id"]}, {"status": 'success'})



def down_tag():
    lists = get_gushiwen_tag()
    for ll in   lists:
        ll['_id'] = ll['href']
        try:
            cidainwang_gushiwen_tag.insertOne(ll)
        except  Exception as e :
            print(e)

if __name__ == "__main__":
    print()
    # down_tag()
    # print(get_gushi())
    down_gushi_list()
    cidainwang_gushiwen.close()
    cidainwang_gushiwen_tag.close()

